/*
 * Created on 4 mai 2005
 * Revised on June 2008
 * TODO To change the template for this generated file go to
 * Window - Preferences - Java - Code Style - Code Templates
 */
package learning;

import ihm.*;
import maze.*;

import java.awt.Point;
import java.util.ArrayList;
import java.util.List;
import maze.SituatedAgent;
import perception.*;

/**
 * @author deloor
 *
 * TODO To change the template for this generated type comment go to
 * Window - Preferences - Java - Code Style - Code Templates
 */
public class SarsaSituatedAgent extends SituatedAgent implements Runnable {

    private PerceptionFactory _perceptionFactory;
    private ArrayList<MemoryPattern> _memory;
    private float _bestQuality = 10;
    private Perception _S;
    private Perception _SP;
    private String _AP;
    private float _alpha = (float) 0.2;
    private float _lambda = (float) 0.2;
    private float _epsilon = (float) 0.5;
    private float _slow = 50;
    private int _nbStep = 1000;
    //private int _nbEpisodeMax = 200;
    private int _nbStepTotal = 0;
    private int _nbEpisode = 0;
    private List<Point> _trace;
    // thread
    private Thread _myThread;
    private boolean running;
    private LearningInMazeView _view;
    private boolean selected;
    private Application _application;
    private String _name;     // son nom, id

    public SarsaSituatedAgent(Application application, LearningInMazeView viewer, Maze m, String name, PerceptionFactory perceptionFactory) {
        super(m);
        _memory = new ArrayList();
        _trace = new ArrayList();
        _application = application;
        _name = name;
        _view = viewer;
        _perceptionFactory = perceptionFactory;

    }

    @Override
    public void init() {
        super.init();

        _S = _perceptionFactory.create();
        _S.setAgent((SituatedAgent) this);
        _S.updatePerception();
    }

    public synchronized void startLife() {
        _myThread = new Thread(this);
        _myThread.start();
        running = true;
    }

    @Override
    public synchronized void run()
    {
        Point p = new Point();
        p = _maze.findAPositionFreeRandomly();
        int nStepTemp = 0;
        int nbEpisode = 0;
        
        while (true) 
        {
            init();
            nStepTemp = 0;
            
            while (nStepTemp < _nbStep) 
            {
                if (isSelected())
                {
                    _view.eraseLearner();
                }
                
                if (_application.getSimulationStatut().equals("run"))
                {
                    sarsaAlgorithmeStep(); // MAJ position et action
                    nStepTemp++;
                    _nbStepTotal++;
                    // Reward de 10 == recompense == gagné == changement de position
                    if (_R == 10) 
                    {
                        p = new Point(nbEpisode++, nStepTemp);
                        _trace.add(p);
                        linearizeTrace(2000);
                        _position = _maze.findAPositionFreeRandomly();
                        // _epsilon= generator.nextFloat();
                        nStepTemp = 0;
                        init();
                        _nbEpisode++;
                        _view.drawGraph();
                        _R = 0;
                        Thread.yield();
                    }
                } else {
                    try {
                        Thread.sleep(10);
                    } catch (InterruptedException e) {
                        
                    }
                }
                
                if (isSelected()) 
                {
                    _view.setNewInformations(this);
                    _view.drawLearner();
                    try {
                        Thread.sleep((int) _slow);
                    } catch (InterruptedException e) {
                    }
                    _view.setNbStep(_nbStepTotal);
                }
            }

            p = new Point(nbEpisode++, nStepTemp);
            nStepTemp = 0;
            _trace.add(p);
            linearizeTrace(2000);
            _nbEpisode++;
            _view.drawGraph();
            if (isSelected()) 
            {
                _view.setNbEpisode(_nbEpisode);
                _view.eraseLearner();
            }
            Thread.yield();
        }
    }

    public void linearizeTrace(int sizeMax)
    {
        if (this._trace.size() > 2 * sizeMax)
        {
            ArrayList newTrace = new ArrayList();
            newTrace.ensureCapacity(sizeMax);
            for (int i = (_trace.size() / 2); i < _trace.size(); i++)
            {
                newTrace.add(_trace.get(i));
            }
            _trace = newTrace;
        }
    }

    /**
     * la "bestQuality" est la valeur la plus élevée de qualité trouvée en mémoire _memory
     */
    @Override
    public float getBestQuality() {
        return _bestQuality;
    }

    /**
     * Cette méthode retourne la qualité la plus élevée pour un état donné
     * Elle recherche toutes les qualités associées à cet état (généralement autant qu'il y
     * a d'actions ...
     */
    @Override
    public float getBestValueForState(Perception aState)
    {
        float value = -1000;
        
        for (MemoryPattern mp : _memory)
        {
            if (mp.getPerception().equals(aState))
            {
                if (mp.getQualitie() > value)
                {
                    value = mp.getQualitie();
                }
            }
        }
        return value;
    }

    public ArrayList<MemoryPattern> getMemory() {
        return _memory;
    }

    public void setEpsilon(float value) {
        _epsilon = value;
        _application.getLearningInMazeView().setNewInformations(this);
    }

    public float getEpsilon() {
        return _epsilon;
    }

    public void setAlpha(float value) {
        _alpha = value;
    }

    public float getAlpha() {
        return _alpha;
    }

    public void setLambda(float value) {
        _lambda = value;
    }

    public float getLambda() {
        return _lambda;
    }

    public void setCompteur(int value) {
        _nbStepTotal = value;
    }

    public int getCompteur() {
        return _nbStepTotal;
    }

    public void setNbEpisode(int value) {
        _nbEpisode = value;
    }

    public int getNbEpisode() {
        return _nbEpisode;
    }

    public void setSlow(long value) {
        _slow = value;
    }

    public float getSlow() {
        return _slow;
    }

    public List<Point> getTrace() {
        return _trace;
    }

    public void createNewMemoryWith(Perception perception) {
        for (String action : _possibleActions) {
            float quality = _randomGenerator.nextFloat() * 10;
            MemoryPattern mp = new MemoryPattern(perception, quality, action);
            _memory.add(mp);
        }
    }

    //init pour d�marrage
    public boolean existeAMemorieWith(Perception state) {
        for (MemoryPattern mp : _memory) {
            Perception lns = mp.getPerception();
            if (lns.equals(state)) {
                return true;
            }
        }
        return false;
    }

    public void learn() 
    {
        float QSA = getQSA(_S, _A);
        float QSAPrime = getQSA(_SP, _AP);
        
        float newQSA = QSA + _alpha * (_R + _lambda * QSAPrime - QSA);
        
        if (newQSA > _bestQuality) {
            _bestQuality = newQSA;
        }
        setQSA(_S, _A, newQSA);
        _S = _SP.copy();
        _A = _AP;
    }

    public void sarsaAlgorithmeStep() {
        runAction();
        chooseAPAction();
        learn(); // recalcul de la qualit� de la case avant d'en changer
    }

    private void setQSA(Perception state, String action, float value) {
        for (MemoryPattern mp : getMemory()) {
            if (mp.getPerception().equals(state) && mp.getAction().equals(action)) {
                mp.setQualitie(value);
                return;
            }
        }
    }

    private float getQSA(Perception perception, String action) {
        float value = 0;
        for (MemoryPattern mp : getMemory())
        {
            if (mp.getPerception().equals(perception) && mp.getAction().equals(action))
            {
                value = mp.getQualitie();
            }
        }
        return value;
    }

    public void chooseAPAction() {
        float choose = _randomGenerator.nextFloat();
        if (choose < _epsilon) {
            chooseAPActionRandomly();
        }
        else {
            chooseAPGreedyAction();
        }
    }

    public void chooseAPActionRandomly() {
        int action = _randomGenerator.nextInt(_possibleActions.size());
        _AP = _possibleActions.get(action);
    }

    public void chooseAPGreedyAction() {
        float q = -100;
        for (MemoryPattern mp : _memory) {
            if (mp.getPerception().equals(_SP)) {
                if (q < mp.getQualitie()) {
                    q = mp.getQualitie();
                    _AP = mp.getAction();
                }
            }

        }
    }

    @Override
    public void runAction() {
        super.runAction(); // retour coord case courante
        _SP = _perceptionFactory.create();
        _SP.setAgent(this);
        _SP.updatePerception();

        if (!(existeAMemorieWith(_SP))) {
            createNewMemoryWith(_SP);
        }
    }

    public String getName() {
        return _name;
    }

    public void setSelected() {
        selected = true;
    }

    public void setUnSelected() {
        selected = false;
    }

    public boolean isSelected() {
        return selected;
    }
}
